{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 加载所需要包\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import jieba\n",
    "import time\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建停用词列表\n",
    "def stopwordslist():\n",
    "    stopwords = [line.strip() for line in open('./cnews/stop.txt',encoding='UTF-8').readlines()]\n",
    "    return stopwords\n",
    "\n",
    "# # 对句子进行中文分词\n",
    "# def seg_depart(sentence):\n",
    "#     # 对文档中的每一行进行中文分词\n",
    "#     print(\"正在分词\")\n",
    "#     sentence_depart = jieba.cut(sentence.strip())\n",
    "#     # 创建一个停用词列表\n",
    "#     stopwords = stopwordslist()\n",
    "#     # 输出结果为outstr\n",
    "#     outstr = ''\n",
    "#     # 去停用词\n",
    "#     for word in sentence_depart:\n",
    "#         if word not in stopwords:\n",
    "#             if word != '\\t':\n",
    "#                 outstr += word\n",
    "#                 outstr += \" \"\n",
    "#     return outstr\n",
    "\n",
    "# # 给出文档路径\n",
    "# filename = \"Init.txt\"\n",
    "# outfilename = \"out.txt\"\n",
    "# inputs = open(filename, 'r', encoding='UTF-8')\n",
    "# outputs = open(outfilename, 'w', encoding='UTF-8')\n",
    "\n",
    "# # 将输出结果写入ou.txt中\n",
    "# for line in inputs:\n",
    "#     line_seg = seg_depart(line)\n",
    "#     outputs.write(line_seg + '\\n')\n",
    "#     print(\"-------------------正在分词和去停用词-----------\")\n",
    "# outputs.close()\n",
    "# inputs.close()\n",
    "# print(\"删除停用词和分词成功！！！\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 1)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 读取停用词\n",
    "# stopword = pd.read_csv(\"./cnews/stop.txt\",\n",
    "#                        quoting=csv.QUOTE_NONE,\n",
    "#                        header=None,\n",
    "#                        names = [\"Stopwords\"])\n",
    "stopword = stopwordslist()\n",
    "## 读取数据集\n",
    "Red_df = pd.read_csv(\"./data/test_data.csv\")\n",
    "## 增大样本量\n",
    "Artical = []\n",
    "for ii in range(20000):\n",
    "    Artical.append(Red_df.content[ii])\n",
    "Red = pd.DataFrame(data = {\"Artical\" : Artical})\n",
    "Red.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:4: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n",
      "  after removing the cwd from sys.path.\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\吨吨吨\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.626 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40.799514199999976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:21: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n"
     ]
    }
   ],
   "source": [
    "## 使用双重for循环分词：\n",
    "\n",
    "## 数据表的行数\n",
    "start = time.clock()\n",
    "row,col = Red.shape\n",
    "## 预定义列表\n",
    "Red[\"cutword\"] = \"cutword\"\n",
    "for ii in np.arange(row):\n",
    "    ## 分词\n",
    "    cutwords = list(jieba.cut(Red.Artical[ii], cut_all=True))\n",
    "    ## 去除长度小于1的词\n",
    "    cutwordii = []\n",
    "    for cut in cutwords:\n",
    "        if len(cut) > 1:\n",
    "            cutwordii.append(cut)\n",
    "    cutwords = pd.Series(cutwordii)\n",
    "    ## 去停用词\n",
    "    cutwords = cutwords[~cutwords.isin(stopword)]\n",
    "    Red.cutword[ii] = cutwords.values\n",
    "\n",
    "end = time.clock()\n",
    "timeforfor = end-start\n",
    "print(timeforfor)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:4: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48.97547359999999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:18: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n"
     ]
    }
   ],
   "source": [
    "## 使用apply在for循环内的方式进行分词：\n",
    "\n",
    "## 数据表的行数\n",
    "start = time.clock()\n",
    "row,col = Red.shape\n",
    "## 预定义列表\n",
    "Red[\"cutword2\"] = \"cutword\"\n",
    "for ii in np.arange(row):\n",
    "    ## 分词\n",
    "    cutwords = list(jieba.cut(Red.Artical[ii], cut_all=True))\n",
    "    ## 去除长度小于1的词\n",
    "    cutwords = pd.Series(cutwords)\n",
    "    cutwords = cutwords[cutwords.apply(len)>1]\n",
    "    ## 去停用词\n",
    "    cutwords = cutwords[~cutwords.isin(stopword)]\n",
    "    Red.cutword2[ii] = cutwords.values\n",
    "\n",
    "end = time.clock()\n",
    "timefor = end-start\n",
    "print(timefor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:16: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35.66855940000005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python\\lib\\site-packages\\ipykernel_launcher.py:19: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n"
     ]
    }
   ],
   "source": [
    "## 只使用apply：\n",
    "\n",
    "def cutword2(string):\n",
    "    cutwords = list(jieba.cut(string, cut_all=True))\n",
    "    ## 去除长度大于1的词\n",
    "    cutwordii = []\n",
    "    for cut in cutwords:\n",
    "        if len(cut) > 1:\n",
    "            cutwordii.append(cut)\n",
    "    cutwords = pd.Series(cutwordii)\n",
    "    ## 去停用词\n",
    "    cutwords = cutwords[~cutwords.isin(stopword)]\n",
    "    return(cutwords.values)\n",
    "\n",
    "## 使用apply方法分词\n",
    "start = time.clock()\n",
    "Red[\"cutword3\"] = \"cutword\"\n",
    "Red[\"cutword3\"] = Red.Artical.apply(func=cutword2)\n",
    "end = time.clock()\n",
    "timeapply = end-start\n",
    "print(timeapply)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Artical</th>\n",
       "      <th>cutword</th>\n",
       "      <th>cutword2</th>\n",
       "      <th>cutword3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>银华旗下两基金开通转换 银华基金日前发布公告，自3月12日起开通银华领先策略、银华增强收益与...</td>\n",
       "      <td>[华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...</td>\n",
       "      <td>[华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...</td>\n",
       "      <td>[华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>深圳楼市现拐点价涨量跌 购房者对房价敏感 据中央电视台“朝闻天下”栏目消息，深圳楼市在今年上...</td>\n",
       "      <td>[深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...</td>\n",
       "      <td>[深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...</td>\n",
       "      <td>[深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>今年以来封基净值平均增长12.46% 本报讯 与开放式基金一样，今年以来封基的表现也很抢眼。...</td>\n",
       "      <td>[今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...</td>\n",
       "      <td>[今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...</td>\n",
       "      <td>[今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>现场：斯科拉扭伤离场检查 洛瑞迟到三分杯水车薪新浪体育讯北京时间2月28日消息，火箭背靠背客...</td>\n",
       "      <td>[现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...</td>\n",
       "      <td>[现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...</td>\n",
       "      <td>[现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>120G硬盘高清DV 索尼XR500E单机8650元    作者：李金昊    [北京行情]...</td>\n",
       "      <td>[120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...</td>\n",
       "      <td>[120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...</td>\n",
       "      <td>[120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>张世礼：社会要尊重设计师劳动 不搞恶性竞争新浪家居杨轶讯 2010年1月22日，新浪乐居•里...</td>\n",
       "      <td>[社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...</td>\n",
       "      <td>[社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...</td>\n",
       "      <td>[社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>搞定Icon的万能尖货Gwen Stefani “娘”了 没救了Gwen Stefani做了...</td>\n",
       "      <td>[搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...</td>\n",
       "      <td>[搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...</td>\n",
       "      <td>[搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>专家称开发商炮制故事助推去年房价本报北京1月6日电(记者李松涛)在某论坛上，国家信息中心专家...</td>\n",
       "      <td>[专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...</td>\n",
       "      <td>[专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...</td>\n",
       "      <td>[专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>吴尊韩庚培养默契 被高晓松关在房里对视聊天成都商报(微博)讯(记者 张世豪 实习生 李潇 摄...</td>\n",
       "      <td>[培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...</td>\n",
       "      <td>[培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...</td>\n",
       "      <td>[培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>根据市场节奏变化选择基金组合在基金组合投资方面，建议投资者继续保持谨慎为主，适度灵活的投资思...</td>\n",
       "      <td>[根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...</td>\n",
       "      <td>[根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...</td>\n",
       "      <td>[根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Artical  \\\n",
       "0      银华旗下两基金开通转换 银华基金日前发布公告，自3月12日起开通银华领先策略、银华增强收益与...   \n",
       "1      深圳楼市现拐点价涨量跌 购房者对房价敏感 据中央电视台“朝闻天下”栏目消息，深圳楼市在今年上...   \n",
       "2      今年以来封基净值平均增长12.46% 本报讯 与开放式基金一样，今年以来封基的表现也很抢眼。...   \n",
       "3      现场：斯科拉扭伤离场检查 洛瑞迟到三分杯水车薪新浪体育讯北京时间2月28日消息，火箭背靠背客...   \n",
       "4      120G硬盘高清DV 索尼XR500E单机8650元    作者：李金昊    [北京行情]...   \n",
       "...                                                  ...   \n",
       "19995  张世礼：社会要尊重设计师劳动 不搞恶性竞争新浪家居杨轶讯 2010年1月22日，新浪乐居•里...   \n",
       "19996  搞定Icon的万能尖货Gwen Stefani “娘”了 没救了Gwen Stefani做了...   \n",
       "19997  专家称开发商炮制故事助推去年房价本报北京1月6日电(记者李松涛)在某论坛上，国家信息中心专家...   \n",
       "19998  吴尊韩庚培养默契 被高晓松关在房里对视聊天成都商报(微博)讯(记者 张世豪 实习生 李潇 摄...   \n",
       "19999  根据市场节奏变化选择基金组合在基金组合投资方面，建议投资者继续保持谨慎为主，适度灵活的投资思...   \n",
       "\n",
       "                                                 cutword  \\\n",
       "0      [华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...   \n",
       "1      [深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...   \n",
       "2      [今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...   \n",
       "3      [现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...   \n",
       "4      [120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...   \n",
       "...                                                  ...   \n",
       "19995  [社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...   \n",
       "19996  [搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...   \n",
       "19997  [专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...   \n",
       "19998  [培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...   \n",
       "19999  [根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...   \n",
       "\n",
       "                                                cutword2  \\\n",
       "0      [华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...   \n",
       "1      [深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...   \n",
       "2      [今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...   \n",
       "3      [现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...   \n",
       "4      [120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...   \n",
       "...                                                  ...   \n",
       "19995  [社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...   \n",
       "19996  [搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...   \n",
       "19997  [专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...   \n",
       "19998  [培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...   \n",
       "19999  [根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...   \n",
       "\n",
       "                                                cutword3  \n",
       "0      [华旗, 旗下, 两基, 基金, 开通, 转换, 基金, 金日, 日前, 发布, 发布公告,...  \n",
       "1      [深圳, 楼市, 拐点, 购房, 购房者, 房价, 敏感, 中央, 中央电视台, 电视, 电...  \n",
       "2      [今年, 今年以来, 以来, 净值, 平均, 增长, 12, 46, 本报, 本报讯, 开放...  \n",
       "3      [现场, 斯科拉, 科拉, 扭伤, 离场, 检查, 迟到, 三分, 杯水, 杯水车薪, 水车...  \n",
       "4      [120G, 硬盘, 高清, DV, 索尼, XR500E, 单机, 8650, 作者, 北...  \n",
       "...                                                  ...  \n",
       "19995  [社会, 尊重, 重设, 设计, 设计师, 劳动, 恶性, 恶性竞争, 竞争, 新浪, 家居...  \n",
       "19996  [搞定, Icon, 万能, Gwen, Stefani, 没救, Gwen, Stefan...  \n",
       "19997  [专家, 开发, 开发商, 炮制, 故事, 助推, 去年, 房价, 本报, 北京, 日电, ...  \n",
       "19998  [培养, 默契, 高晓松, 关在, 房里, 对视, 聊天, 天成, 成都, 商报, 记者, ...  \n",
       "19999  [根据, 市场, 节奏, 变化, 选择, 基金, 组合, 合在, 基金, 组合, 投资, 投...  \n",
       "\n",
       "[20000 rows x 4 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Red"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
